# update.py: Scrape all data from schedule.berkeley.edu.  Add or update entries.

from django.core.management.base import BaseCommand, CommandError
from alpha.models import Building
from alpha.models import Room
from alpha.models import TimeBlock
from alpha.models import SectionTimeBlock
from alpha.models import Major
from alpha.models import Course
from alpha.models import Section
import class_parser
import urllib
import re
import datetime
import Queue
import threading
import sys

time_re = re.compile('([0-9]{1,4})-([0-9]{1,4})([PA])')
catalog_re = re.compile('<TD><FONT SIZE="-1"><B>Description: </B>(.*?)</FONT></TD>')
GENERAL_CATALOG_URL = 'http://osoc.berkeley.edu/catalog/gcc_search_sends_request'

DAYS = {
  'M':  0,
  'Tu': 1,
  'W':  2,
  'Th': 3,
  'F':  4,
  'SA': 5,
  'Su': 6,
}

class Counter(object):
  def __init__(self):
    self.lock = threading.Lock()
    self.count = 0
    
  def inc(self):
    with self.lock:
      self.count += 1
     
  def cur(self):
    with self.lock:
      return self.count


class Command(BaseCommand):
  def handle(self, *args, **options):
    # Start with the root page, which is the page of all classes
    print 'Retriving all classes from %s...' % class_parser.ROOT_URL,
    initial_root = class_parser.get_root(class_parser.ROOT_URL)
    rows = class_parser.get_entries(initial_root)
    print 'done'
    # Note: majors is a list in the format (shortname, longname)
    # Courses is in the format (major_shortname, shortname, longname, url)
    majors, courses = class_parser.process_trs(rows)
    
    # Now that we have the majors, lets store them in the database
    print 'Saving majors in database (%d entries)...' % len(majors),
    new_entries = 0
    for major_shortname, major_longname in majors:
      # print '%s (%s)' % (major_shortname, major_longname)
      major, created = Major.objects.get_or_create(shortname=major_shortname)
      if created: 
        new_entries += 1
        major.longname = major_longname
        major.save()
    print 'done [%d entries created]' % new_entries
    
    # In the meantime, we can also store all the classes information in the db
    print 'Saving courses in database (%d entries)...' % len(courses)
    new_entries = 0
    count = 0
    for major_shortname, code, title, url in courses:
      count += 1
      print '(%4d)%s %s - %s' % (count, major_shortname, code, title),
      major = Major.objects.get(shortname=major_shortname)
      course, created = Course.objects.get_or_create(code=code, major=major,
          defaults={'title': title})
      if created:
        print '\tCreated',
        new_entries += 1
        course.save()
      if not course.description and False:  # TODO: REMOVE THE FALSE IF WE WANT DESCRIPTION DATA
        # Fetch a description from the general catalog
        data = {'p_dept_cd': major_shortname, 'p_title': '', 'p_number': code}
        try:
          u = urllib.urlopen(GENERAL_CATALOG_URL, urllib.urlencode(data))
          print '\tDescription',
        except Exception:
          pass
        else:
          result = catalog_re.search(u.read())
          if result:
            course.description = result.group(1)
          if not result or not course.description:
            course.description = 'N/A'
          course.save()
      print
    print 'done [%d entries created]' % new_entries
    
    # Wrote a routine to store a piece of section data so we can do it as we
    # Fetch data
    def store_section_data(section_data, time_lock, room_lock, section_lock):
      # Prepare data
      (major_shortname, code, section_type, section_number, parent, 
          section_time, section_location, instructor, ccn,  last_changed,
          book_url, primary) = section_data
      print '\t%s %s - %s %s (%s)' % (major_shortname, code, section_number, section_type, section_location)
      # print section_data
      try:
        major = Major.objects.get(shortname=major_shortname)
        course = Course.objects.get(major=major, code=code)
      except Exception:  # Does not exist
        return
      try:
        ccn = int(ccn)
      except Exception:  # TODO: Use a more specific exception?
        ccn = None
      if section_location:
        cleaned_location_groups = re.search('([0-9A-Za-z ]+)', section_location)
        cleaned_location = cleaned_location_groups.group(1)
        location_groups = re.search('([A-Z]?[0-9]+[A-Z]?) (.+)', cleaned_location)
        if location_groups:
          room_name, building_name = location_groups.groups()
          room_name = room_name.strip()
          building_name = building_name.strip()
        elif cleaned_location == 'WHEELER AUD':  # Special Case
          building_name = 'WHEELER'
          room_name = 'AUD'
        else:
          building_name = cleaned_location.strip()
          room_name = 'N/A'
        with room_lock:
          building, created = Building.objects.get_or_create(shortname=building_name)
          if created: building.save()
          room, created = Room.objects.get_or_create(building=building, name=room_name)
          if created: room.save()
      else:
        with room_lock:
          building, created = Building.objects.get_or_create(shortname='N/A')
          if created: building.save()
          room_name = 'N/A'
          room, created = Room.objects.get_or_create(building=building, name=room_name)
          if created: room.save()
        
      # Prepare time blocks
      time_blocks = []
      section_time_blocks = []
      if section_time and len(section_time.split(' ')) > 1:
        # Is in the format of Tu 7-8P, or MWF 930-11A
        days, times = section_time.split(' ')
        days_groups = re.findall('(M|Tu|W|Th|F|SA|Su)', days)
        if not days_groups:
          return
        time_groups = time_re.search(times)
        start_time_str, end_time_str, am_pm = time_groups.groups()
        
        # Create python datetime objects.  We use datetime because time doesn't
        # support add/sub operations with timedelta objects
        if len(start_time_str) > 2:
          if len(start_time_str) == 3:
            start_time_hour = int(start_time_str[0])
          else:
            start_time_hour = int(start_time_str[:2])
          start_time_minutes = int(start_time_str[-2:])
          start_time = datetime.datetime(2000, 1, 1, start_time_hour, start_time_minutes)
        else:
          start_time_hour = int(start_time_str)
          start_time_minutes = 0
          start_time = datetime.datetime(2000, 1, 1, start_time_hour)

        if len(end_time_str) > 2:
          if len(end_time_str) == 3:
            end_time_hour = int(end_time_str[0])
          else:
            end_time_hour = int(end_time_str[:2])
          end_time_minutes = int(end_time_str[-2:])
          end_time = datetime.datetime(2000, 1, 1, end_time_hour, end_time_minutes)
        else:
          end_time_hour = int(end_time_str)
          end_time_minutes = 0
          end_time = datetime.datetime(2000, 1, 1, end_time_hour)

        # Add 12 hours if its PM, unless its 12PM
        if am_pm == 'P' and end_time_hour != 12:
          half_day_delta = datetime.timedelta(hours=12)
          end_time = end_time + half_day_delta
          # Only add the half day delta if start_time < end_time
          if start_time_hour < end_time_hour or start_time_hour == 12:
            start_time = start_time + half_day_delta
        
        # Create time blocks and section time blocks
        half_hour = datetime.timedelta(minutes=30)
        for day in days_groups:
          current_start_time = start_time
          day_code = DAYS[day]  # Convert MTuWThF to database format (0, 1, ...)
          # We need to keep track of consecutive time blocks
          todays_time_blocks = []
          # We're going to destructively loop over start time
          while current_start_time < end_time:
            block_end_time = current_start_time + half_hour
            with time_lock:
              time_block, created = TimeBlock.objects.get_or_create(day=day_code, 
                  start_time=current_start_time.time(), end_time=block_end_time.time())
              if created:
                time_block.description = '%s %s-%s' % (time_block.get_day_display(),
                    current_start_time.strftime('%I:%M%p'), block_end_time.strftime('%I:%M%p'))
                # print 'New time block ', time_block.description 
                time_block.save()
            # Add time block into section, then increment
            time_blocks.append(time_block)
            todays_time_blocks.append(time_block)
            current_start_time = current_start_time + half_hour
          # Create a new section time block for all the consecutive time blocks we have
          with time_lock:
            section_time_block, created = SectionTimeBlock.objects.get_or_create(
                start_time=start_time, end_time=end_time, day=day_code, room=room)
            if created:
              section_time_block.description = '%s %s' % (day, times)
            for time_block in todays_time_blocks:
              section_time_block.time_blocks.add(time_block)
            section_time_block.save()
          section_time_blocks.append(section_time_block)
      
      # Create section
      with section_lock:
        section, created = Section.objects.get_or_create(course=course, 
            number=section_number, type=section_type, defaults={
            'ccn': ccn,
            'building': building,
            'room': room,
            'time': section_time or 'N/A',
            'instructor': instructor
            })
        for time_block in time_blocks:
          section.time_blocks.add(time_block)
        for section_time_block in section_time_blocks:
          section.section_time_blocks.add(section_time_block)
        if created:
          section.save()
    
    # Now we need to actually fetch section data
    
    def _process_thread(queue, counter, total, time_lock, room_lock, section_lock):
      while True:
        try:
          major, short, long, url = queue.get(False)
          counter.inc()
          print '%s %s\t(%2d%%)' % (major, short, (counter.cur()*100/total))
          datas = class_parser.process_class_page(url)
          for data in datas:
            course, primary, section_number, section_type = class_parser.parse_course(data['Course:'])
            section_time, section_location = class_parser.parse_timeloc(data['Location:'])
            section_data = [
                major,
                short,
                section_type,
                section_number,
                None,  # Parent isn't useful to us
                section_time,
                section_location,
                data['Instructor:'],
                data['Course Control Number:'],
                data['Status/Last Changed:'],
                None,  # Book url is not important
                None,  # Primary is not important
            ]
            store_section_data(section_data, time_lock, room_lock, section_lock)
        except Queue.Empty:
          return
      
    def run_get_classes(courses, num_threads):
      q = Queue.Queue()
      threads = []
      counter = Counter()
      total = len(courses)
      time_lock = threading.Lock()
      room_lock = threading.Lock()
      section_lock = threading.Lock()
      for item in courses:
        major, short, long, url = item
        course = Course.objects.get(major__shortname=major, code=short)
        if not Section.objects.filter(course=course):
          # Only queue it in if a section with the same course doesn't yet exist
          q.put(item)  
      for i in range(0, num_threads):
        t = threading.Thread(target=_process_thread, 
            args=[q, counter, total, time_lock, room_lock, section_lock])
        t.start()
        threads.append(t)
      for t in threads:
        t.join()
      
    print 'Fetching sections data...',
    run_get_classes(courses, 30)
    print 'done'
      